{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LightGBM\n",
    "\n",
    "[LightGBM——提升机器算法](https://blog.csdn.net/huacha__/article/details/81057150?utm_medium=distribute.pc_relevant.none-task-blog-BlogCommendFromMachineLearnPai2-1.channel_param&depth_1-utm_source=distribute.pc_relevant.none-task-blog-BlogCommendFromMachineLearnPai2-1.channel_param   \"LightGBM——提升机器算法\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "System   version:  3.6.10 (default, Mar  5 2020, 10:17:47) [MSC v.1900 64 bit (AMD64)]\n",
      "LightGBM version:  3.0.0\n",
      "numpy  Version: 1.16.0\n",
      "pandas Version: 1.0.4\n"
     ]
    }
   ],
   "source": [
    "import os, sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import lightgbm as lgb\n",
    "import category_encoders as ce\n",
    "from sklearn.metrics import roc_auc_score, log_loss\n",
    "\n",
    "print(\"System   version:  {}\".format(sys.version))\n",
    "print(\"LightGBM version:  {}\".format(lgb.__version__))\n",
    "print('numpy  Version: {}'.format(np.__version__))\n",
    "print('pandas Version: {}'.format(pd.__version__))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data Shape: (100000, 40)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Label</th>\n",
       "      <th>I1</th>\n",
       "      <th>I2</th>\n",
       "      <th>I3</th>\n",
       "      <th>I4</th>\n",
       "      <th>I5</th>\n",
       "      <th>I6</th>\n",
       "      <th>I7</th>\n",
       "      <th>I8</th>\n",
       "      <th>I9</th>\n",
       "      <th>...</th>\n",
       "      <th>C17</th>\n",
       "      <th>C18</th>\n",
       "      <th>C19</th>\n",
       "      <th>C20</th>\n",
       "      <th>C21</th>\n",
       "      <th>C22</th>\n",
       "      <th>C23</th>\n",
       "      <th>C24</th>\n",
       "      <th>C25</th>\n",
       "      <th>C26</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1382.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>181.0</td>\n",
       "      <td>...</td>\n",
       "      <td>e5ba7672</td>\n",
       "      <td>f54016b9</td>\n",
       "      <td>21ddcdc9</td>\n",
       "      <td>b1252a9d</td>\n",
       "      <td>07b5194c</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3a171ecb</td>\n",
       "      <td>c5c50484</td>\n",
       "      <td>e8b83407</td>\n",
       "      <td>9727dd16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>07c540c4</td>\n",
       "      <td>b04e4670</td>\n",
       "      <td>21ddcdc9</td>\n",
       "      <td>5840adea</td>\n",
       "      <td>60f6221e</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3a171ecb</td>\n",
       "      <td>43f13e8b</td>\n",
       "      <td>e8b83407</td>\n",
       "      <td>731c3655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>767.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>245.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8efede7f</td>\n",
       "      <td>3412118d</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>e587c466</td>\n",
       "      <td>ad3062eb</td>\n",
       "      <td>3a171ecb</td>\n",
       "      <td>3b183c5c</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>893</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4392.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1e88c74f</td>\n",
       "      <td>74ef3502</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6b3a5ca6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3a171ecb</td>\n",
       "      <td>9117a34a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1e88c74f</td>\n",
       "      <td>26b3c7a7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21c9516a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32c7478e</td>\n",
       "      <td>b34f3128</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Label   I1   I2    I3    I4      I5    I6    I7   I8     I9  ...       C17  \\\n",
       "0      0  1.0    1   5.0   0.0  1382.0   4.0  15.0  2.0  181.0  ...  e5ba7672   \n",
       "1      0  2.0    0  44.0   1.0   102.0   8.0   2.0  2.0    4.0  ...  07c540c4   \n",
       "2      0  2.0    0   1.0  14.0   767.0  89.0   4.0  2.0  245.0  ...  8efede7f   \n",
       "3      0  NaN  893   NaN   NaN  4392.0   NaN   0.0  0.0    0.0  ...  1e88c74f   \n",
       "4      0  3.0   -1   NaN   0.0     2.0   0.0   3.0  0.0    0.0  ...  1e88c74f   \n",
       "\n",
       "        C18       C19       C20       C21       C22       C23       C24  \\\n",
       "0  f54016b9  21ddcdc9  b1252a9d  07b5194c       NaN  3a171ecb  c5c50484   \n",
       "1  b04e4670  21ddcdc9  5840adea  60f6221e       NaN  3a171ecb  43f13e8b   \n",
       "2  3412118d       NaN       NaN  e587c466  ad3062eb  3a171ecb  3b183c5c   \n",
       "3  74ef3502       NaN       NaN  6b3a5ca6       NaN  3a171ecb  9117a34a   \n",
       "4  26b3c7a7       NaN       NaN  21c9516a       NaN  32c7478e  b34f3128   \n",
       "\n",
       "        C25       C26  \n",
       "0  e8b83407  9727dd16  \n",
       "1  e8b83407  731c3655  \n",
       "2       NaN       NaN  \n",
       "3       NaN       NaN  \n",
       "4       NaN       NaN  \n",
       "\n",
       "[5 rows x 40 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  criteo 数据集\n",
    "filepath = './data/dac/dac_sample.txt'\n",
    "\n",
    "nume_cols = [\"I\" + str(i) for i in range(1, 14)]\n",
    "cate_cols = [\"C\" + str(i) for i in range(1, 27)]\n",
    "label_col = \"Label\"\n",
    "\n",
    "header = [label_col] + nume_cols + cate_cols\n",
    "\n",
    "all_data = pd.read_csv(filepath, sep=\"\\t\", header=None, names=header)\n",
    "\n",
    "print('Data Shape:', all_data.shape)\n",
    "all_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "ord_encoder = ce.ordinal.OrdinalEncoder(cols=cate_cols)\n",
    "\n",
    "def encode_csv(df, encoder, label_col, typ='fit'):\n",
    "    if typ == 'fit':\n",
    "        df = encoder.fit_transform(df)\n",
    "    else:\n",
    "        df = encoder.transform(df)\n",
    "    y = df[label_col].values\n",
    "    del df[label_col]\n",
    "    return df, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split data to 3 sets\n",
    "length = len(all_data)\n",
    "train_data = all_data.loc[:0.8*length-1]\n",
    "valid_data = all_data.loc[0.8*length:0.9*length-1]\n",
    "test_data = all_data.loc[0.9*length:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Data Shape: X: (80000, 39); Y: (80000,).\n",
      "Valid Data Shape: X: (10000, 39); Y: (10000,).\n",
      "Test Data Shape: X: (10000, 39); Y: (10000,).\n",
      "\n",
      "    I1   I2    I3    I4      I5    I6    I7   I8     I9  I10  ...  C17  C18  \\\n",
      "0  1.0    1   5.0   0.0  1382.0   4.0  15.0  2.0  181.0  1.0  ...    1    1   \n",
      "1  2.0    0  44.0   1.0   102.0   8.0   2.0  2.0    4.0  1.0  ...    2    2   \n",
      "2  2.0    0   1.0  14.0   767.0  89.0   4.0  2.0  245.0  1.0  ...    3    3   \n",
      "3  NaN  893   NaN   NaN  4392.0   NaN   0.0  0.0    0.0  NaN  ...    4    4   \n",
      "4  3.0   -1   NaN   0.0     2.0   0.0   3.0  0.0    0.0  1.0  ...    4    5   \n",
      "\n",
      "   C19  C20  C21  C22  C23  C24  C25  C26  \n",
      "0    1    1    1    1    1    1    1    1  \n",
      "1    1    2    2    1    1    2    1    2  \n",
      "2    2    3    3    2    1    3    2    3  \n",
      "3    2    3    4    1    1    4    2    3  \n",
      "4    2    3    5    1    2    5    2    3  \n",
      "\n",
      "[5 rows x 39 columns]\n"
     ]
    }
   ],
   "source": [
    "# 数据特征编码转换\n",
    "train_x, train_y = encode_csv(train_data, ord_encoder, label_col)\n",
    "valid_x, valid_y = encode_csv(valid_data, ord_encoder, label_col, 'transform')\n",
    "test_x, test_y = encode_csv(test_data, ord_encoder, label_col, 'transform')\n",
    "\n",
    "print('Train Data Shape: X: {trn_x_shape}; Y: {trn_y_shape}.\\nValid Data Shape: X: {vld_x_shape}; Y: {vld_y_shape}.\\nTest Data Shape: X: {tst_x_shape}; Y: {tst_y_shape}.\\n'\n",
    "      .format(trn_x_shape=train_x.shape,\n",
    "              trn_y_shape=train_y.shape,\n",
    "              vld_x_shape=valid_x.shape,\n",
    "              vld_y_shape=valid_y.shape,\n",
    "              tst_x_shape=test_x.shape,\n",
    "              tst_y_shape=test_y.shape,))\n",
    "\n",
    "print(train_x.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LEAF = 64\n",
    "MIN_DATA = 20\n",
    "NUM_OF_TREES = 100\n",
    "TREE_LEARNING_RATE = 0.15\n",
    "EARLY_STOPPING_ROUNDS = 20\n",
    "METRIC = \"auc\"\n",
    "SIZE = \"sample\"\n",
    "\n",
    "params = {\n",
    "    'task': 'train',\n",
    "    'boosting_type': 'gbdt',\n",
    "    'num_class': 1,\n",
    "    'objective': \"binary\",\n",
    "    'metric': METRIC,\n",
    "    'num_leaves': MAX_LEAF,\n",
    "    'min_data': MIN_DATA,\n",
    "    'boost_from_average': True,\n",
    "    #set it according to your cpu cores.\n",
    "    'num_threads': 20,\n",
    "    'feature_fraction': 0.8,\n",
    "    'learning_rate': TREE_LEARNING_RATE,\n",
    "}\n",
    "\n",
    "lgb_train = lgb.Dataset(train_x, train_y.reshape(-1), params=params, categorical_feature=cate_cols)\n",
    "lgb_valid = lgb.Dataset(valid_x, valid_y.reshape(-1), reference=lgb_train, categorical_feature=cate_cols)\n",
    "lgb_test = lgb.Dataset(test_x, test_y.reshape(-1), reference=lgb_train, categorical_feature=cate_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Info] Number of positive: 17958, number of negative: 62042\n",
      "[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.052983 seconds.\n",
      "You can set `force_col_wise=true` to remove the overhead.\n",
      "[LightGBM] [Info] Total Bins 38971\n",
      "[LightGBM] [Info] Number of data points in the train set: 80000, number of used features: 39\n",
      "[LightGBM] [Info] "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\envs\\py36_B\\lib\\site-packages\\lightgbm\\basic.py:1286: UserWarning: Overriding the parameters from Reference Dataset.\n",
      "  warnings.warn('Overriding the parameters from Reference Dataset.')\n",
      "C:\\ProgramData\\Anaconda3\\envs\\py36_B\\lib\\site-packages\\lightgbm\\basic.py:1098: UserWarning: categorical_column in param dict is overridden.\n",
      "  warnings.warn('{} in param dict is overridden.'.format(cat_alias))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[binary:BoostFromScore]: pavg=0.224475 -> initscore=-1.239776\n",
      "[LightGBM] [Info] Start training from score -1.239776\n",
      "[1]\tvalid_0's auc: 0.723997\n",
      "Training until validation scores don't improve for 20 rounds\n",
      "[2]\tvalid_0's auc: 0.736596\n",
      "[3]\tvalid_0's auc: 0.740862\n",
      "[4]\tvalid_0's auc: 0.745639\n",
      "[5]\tvalid_0's auc: 0.74944\n",
      "[6]\tvalid_0's auc: 0.751076\n",
      "[7]\tvalid_0's auc: 0.752973\n",
      "[8]\tvalid_0's auc: 0.753746\n",
      "[9]\tvalid_0's auc: 0.7546\n",
      "[10]\tvalid_0's auc: 0.755247\n",
      "[11]\tvalid_0's auc: 0.7565\n",
      "[12]\tvalid_0's auc: 0.756926\n",
      "[13]\tvalid_0's auc: 0.757582\n",
      "[14]\tvalid_0's auc: 0.757758\n",
      "[15]\tvalid_0's auc: 0.758596\n",
      "[16]\tvalid_0's auc: 0.75866\n",
      "[17]\tvalid_0's auc: 0.759058\n",
      "[18]\tvalid_0's auc: 0.759658\n",
      "[19]\tvalid_0's auc: 0.759656\n",
      "[20]\tvalid_0's auc: 0.758798\n",
      "[21]\tvalid_0's auc: 0.759329\n",
      "[22]\tvalid_0's auc: 0.759307\n",
      "[23]\tvalid_0's auc: 0.759538\n",
      "[24]\tvalid_0's auc: 0.758459\n",
      "[25]\tvalid_0's auc: 0.758257\n",
      "[26]\tvalid_0's auc: 0.758256\n",
      "[27]\tvalid_0's auc: 0.757745\n",
      "[28]\tvalid_0's auc: 0.757413\n",
      "[29]\tvalid_0's auc: 0.757701\n",
      "[30]\tvalid_0's auc: 0.757612\n",
      "[31]\tvalid_0's auc: 0.757312\n",
      "[32]\tvalid_0's auc: 0.75732\n",
      "[33]\tvalid_0's auc: 0.756654\n",
      "[34]\tvalid_0's auc: 0.756381\n",
      "[35]\tvalid_0's auc: 0.756279\n",
      "[36]\tvalid_0's auc: 0.756333\n",
      "[37]\tvalid_0's auc: 0.756346\n",
      "[38]\tvalid_0's auc: 0.756381\n",
      "Early stopping, best iteration is:\n",
      "[18]\tvalid_0's auc: 0.759658\n"
     ]
    }
   ],
   "source": [
    "# 模型训练\n",
    "lgb_model = lgb.train(params,\n",
    "                      lgb_train,\n",
    "                      num_boost_round=NUM_OF_TREES,\n",
    "                      early_stopping_rounds=EARLY_STOPPING_ROUNDS,\n",
    "                      valid_sets=lgb_valid,\n",
    "                      categorical_feature=cate_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'auc': 0.7655408801711783, 'logloss': 0.4682583178835999}\n"
     ]
    }
   ],
   "source": [
    "# 测试效果\n",
    "test_preds = lgb_model.predict(test_x)\n",
    "auc = roc_auc_score(np.asarray(test_y.reshape(-1)), np.asarray(test_preds))\n",
    "logloss = log_loss(np.asarray(test_y.reshape(-1)), np.asarray(test_preds), eps=1e-12)\n",
    "res_basic = {\"auc\": auc, \"logloss\": logloss}\n",
    "print(res_basic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<lightgbm.basic.Booster at 0x1ca3790ae80>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 保存模型\n",
    "tmp = './_model'\n",
    "os.makedirs(tmp, exist_ok=True)\n",
    "save_file = os.path.join(tmp, 'finished.model')\n",
    "lgb_model.save_model(save_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'auc': 0.7655408801711783, 'logloss': 0.4682583178835999}\n"
     ]
    }
   ],
   "source": [
    "# 加载模型\n",
    "loaded_model = lgb.Booster(model_file=save_file)\n",
    "# eval the performance again\n",
    "test_preds = loaded_model.predict(test_x)\n",
    "\n",
    "auc = roc_auc_score(np.asarray(test_y.reshape(-1)), np.asarray(test_preds))\n",
    "logloss = log_loss(np.asarray(test_y.reshape(-1)), np.asarray(test_preds), eps=1e-12)\n",
    "res_basic = {\"auc\": auc, \"logloss\": logloss}\n",
    "print(res_basic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "###  数据处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-10-13 00:27:44,325 [INFO] Filtering and fillna features\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 26/26 [00:02<00:00,  8.78it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 13/13 [00:00<00:00, 252.44it/s]\n",
      "2020-10-13 00:27:47,348 [INFO] Ordinal encoding cate features\n",
      "2020-10-13 00:27:49,167 [INFO] Target encoding cate features\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 26/26 [00:08<00:00,  3.06it/s]\n",
      "2020-10-13 00:27:57,658 [INFO] Start manual binary encoding\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 65/65 [00:04<00:00, 13.35it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 26/26 [00:04<00:00,  6.12it/s]\n",
      "2020-10-13 00:28:07,094 [INFO] Filtering and fillna features\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 26/26 [00:00<00:00, 81.63it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 13/13 [00:00<00:00, 1238.24it/s]\n",
      "2020-10-13 00:28:07,433 [INFO] Ordinal encoding cate features\n",
      "2020-10-13 00:28:07,611 [INFO] Target encoding cate features\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 26/26 [00:01<00:00, 21.15it/s]\n",
      "2020-10-13 00:28:08,846 [INFO] Start manual binary encoding\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 65/65 [00:03<00:00, 19.43it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 26/26 [00:01<00:00, 16.54it/s]\n",
      "2020-10-13 00:28:13,935 [INFO] Filtering and fillna features\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 26/26 [00:00<00:00, 88.26it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 13/13 [00:00<00:00, 1444.43it/s]\n",
      "2020-10-13 00:28:14,249 [INFO] Ordinal encoding cate features\n",
      "2020-10-13 00:28:14,447 [INFO] Target encoding cate features\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 26/26 [00:01<00:00, 25.32it/s]\n",
      "2020-10-13 00:28:15,480 [INFO] Start manual binary encoding\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 65/65 [00:03<00:00, 17.76it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 26/26 [00:01<00:00, 15.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Data Shape: X: (80000, 268); Y: (80000, 1).\n",
      "Valid Data Shape: X: (10000, 268); Y: (10000, 1).\n",
      "Test Data Shape: X: (10000, 268); Y: (10000, 1).\n",
      "\n",
      "[  2.           0.          44.           1.         102.\n",
      "   8.           2.           2.           4.           1.\n",
      "   1.           0.93903309   4.           0.           1.\n",
      "   0.           0.           0.           0.           0.\n",
      "   0.           0.           1.           0.           0.\n",
      "   0.           0.           0.           0.           0.\n",
      "   1.           0.           0.           0.           0.\n",
      "   0.           0.           0.           0.           0.\n",
      "   0.           0.           0.           0.           0.\n",
      "   0.           0.           0.           0.           0.\n",
      "   1.           0.           0.           0.           0.\n",
      "   0.           1.           0.           1.           0.\n",
      "   0.           0.           1.           0.           0.\n",
      "   1.           0.           0.           0.           0.\n",
      "   0.           0.           0.           1.           0.\n",
      "   0.           0.           0.           0.           0.\n",
      "   0.           0.           1.           0.           0.\n",
      "   0.           0.           0.           0.           0.\n",
      "   0.           0.           1.           0.           0.\n",
      "   0.           0.           0.           0.           0.\n",
      "   0.           1.           0.           0.           0.\n",
      "   0.           0.           0.           1.           0.\n",
      "   0.           0.           1.           0.           0.\n",
      "   0.           0.           0.           0.           0.\n",
      "   0.           0.           0.           1.           0.\n",
      "   0.           0.           0.           1.           0.\n",
      "   0.           1.           0.           0.           0.\n",
      "   0.           0.           0.           0.           0.\n",
      "   0.           0.           1.           0.           0.\n",
      "   0.           0.           0.           0.           0.\n",
      "   0.           0.           0.           1.           0.\n",
      "   0.           0.           0.           0.           0.\n",
      "   0.           0.           0.           1.           0.\n",
      "   0.           0.           0.           0.           0.\n",
      "   0.           0.           0.           0.           1.\n",
      "   0.           0.           0.           0.           1.\n",
      "   0.           0.           0.           0.           0.\n",
      "   0.           0.           0.           0.           0.\n",
      "   1.           0.           0.           0.           0.\n",
      "   0.           0.           0.           0.           0.\n",
      "   1.           0.           0.           0.           1.\n",
      "   0.           0.           0.           0.           0.\n",
      "   0.           0.           0.           1.           0.\n",
      "   0.           0.           0.           0.           0.\n",
      "   0.           0.           0.           1.           0.\n",
      "   0.           1.           0.           0.           0.\n",
      "   0.           0.           0.           0.           0.\n",
      "   1.           0.           0.           0.           1.\n",
      "   0.           0.           0.           0.           1.\n",
      "   0.           0.           0.           0.           0.\n",
      "   0.           0.           0.           1.           0.\n",
      "   0.           0.           0.           0.           0.\n",
      "   1.           0.           0.           0.           0.\n",
      "   0.           0.           0.        ]\n",
      "[0]\n"
     ]
    }
   ],
   "source": [
    "import lgb_model.lightgbm_utils as lgb_utils\n",
    "label_col = 'Label'\n",
    "num_encoder = lgb_utils.NumEncoder(cate_cols, nume_cols, label_col)\n",
    "train_x, train_y = num_encoder.fit_transform(train_data)\n",
    "valid_x, valid_y = num_encoder.transform(valid_data)\n",
    "test_x, test_y = num_encoder.transform(test_data)\n",
    "del num_encoder\n",
    "print('Train Data Shape: X: {trn_x_shape}; Y: {trn_y_shape}.\\nValid Data Shape: X: {vld_x_shape}; Y: {vld_y_shape}.\\nTest Data Shape: X: {tst_x_shape}; Y: {tst_y_shape}.\\n'\n",
    "      .format(trn_x_shape=train_x.shape,\n",
    "              trn_y_shape=train_y.shape,\n",
    "              vld_x_shape=valid_x.shape,\n",
    "              vld_y_shape=valid_y.shape,\n",
    "              tst_x_shape=test_x.shape,\n",
    "              tst_y_shape=test_y.shape,))\n",
    "\n",
    "print(train_x[1,:])\n",
    "print(train_y[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Info] Number of positive: 17958, number of negative: 62042\n",
      "[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.296578 seconds.\n",
      "You can set `force_col_wise=true` to remove the overhead.\n",
      "[LightGBM] [Info] Total Bins 15787\n",
      "[LightGBM] [Info] Number of data points in the train set: 80000, number of used features: 267\n",
      "[LightGBM] [Info] [binary:BoostFromScore]: pavg=0.224475 -> initscore=-1.239776\n",
      "[LightGBM] [Info] Start training from score -1.239776\n",
      "[1]\tvalid_0's auc: 0.727035\n",
      "Training until validation scores don't improve for 20 rounds\n",
      "[2]\tvalid_0's auc: 0.745243\n",
      "[3]\tvalid_0's auc: 0.749993\n",
      "[4]\tvalid_0's auc: 0.750781\n",
      "[5]\tvalid_0's auc: 0.753203\n",
      "[6]\tvalid_0's auc: 0.754548\n",
      "[7]\tvalid_0's auc: 0.756143\n",
      "[8]\tvalid_0's auc: 0.757718\n",
      "[9]\tvalid_0's auc: 0.758371\n",
      "[10]\tvalid_0's auc: 0.759261\n",
      "[11]\tvalid_0's auc: 0.760498\n",
      "[12]\tvalid_0's auc: 0.761402\n",
      "[13]\tvalid_0's auc: 0.762118\n",
      "[14]\tvalid_0's auc: 0.762916\n",
      "[15]\tvalid_0's auc: 0.764051\n",
      "[16]\tvalid_0's auc: 0.764829\n",
      "[17]\tvalid_0's auc: 0.76559\n",
      "[18]\tvalid_0's auc: 0.766304\n",
      "[19]\tvalid_0's auc: 0.766337\n",
      "[20]\tvalid_0's auc: 0.767213\n",
      "[21]\tvalid_0's auc: 0.767702\n",
      "[22]\tvalid_0's auc: 0.767753\n",
      "[23]\tvalid_0's auc: 0.768076\n",
      "[24]\tvalid_0's auc: 0.768541\n",
      "[25]\tvalid_0's auc: 0.768448\n",
      "[26]\tvalid_0's auc: 0.769026\n",
      "[27]\tvalid_0's auc: 0.769197\n",
      "[28]\tvalid_0's auc: 0.769597\n",
      "[29]\tvalid_0's auc: 0.769688\n",
      "[30]\tvalid_0's auc: 0.769907\n",
      "[31]\tvalid_0's auc: 0.769919\n",
      "[32]\tvalid_0's auc: 0.769846\n",
      "[33]\tvalid_0's auc: 0.77005\n",
      "[34]\tvalid_0's auc: 0.770085\n",
      "[35]\tvalid_0's auc: 0.77021\n",
      "[36]\tvalid_0's auc: 0.770272\n",
      "[37]\tvalid_0's auc: 0.76992\n",
      "[38]\tvalid_0's auc: 0.770547\n",
      "[39]\tvalid_0's auc: 0.770695\n",
      "[40]\tvalid_0's auc: 0.77068\n",
      "[41]\tvalid_0's auc: 0.770748\n",
      "[42]\tvalid_0's auc: 0.770631\n",
      "[43]\tvalid_0's auc: 0.77085\n",
      "[44]\tvalid_0's auc: 0.770606\n",
      "[45]\tvalid_0's auc: 0.770498\n",
      "[46]\tvalid_0's auc: 0.770594\n",
      "[47]\tvalid_0's auc: 0.770367\n",
      "[48]\tvalid_0's auc: 0.770168\n",
      "[49]\tvalid_0's auc: 0.770126\n",
      "[50]\tvalid_0's auc: 0.770045\n",
      "[51]\tvalid_0's auc: 0.769959\n",
      "[52]\tvalid_0's auc: 0.770195\n",
      "[53]\tvalid_0's auc: 0.770192\n",
      "[54]\tvalid_0's auc: 0.770075\n",
      "[55]\tvalid_0's auc: 0.770156\n",
      "[56]\tvalid_0's auc: 0.769972\n",
      "[57]\tvalid_0's auc: 0.769837\n",
      "[58]\tvalid_0's auc: 0.769985\n",
      "[59]\tvalid_0's auc: 0.770135\n",
      "[60]\tvalid_0's auc: 0.770282\n",
      "[61]\tvalid_0's auc: 0.770291\n",
      "[62]\tvalid_0's auc: 0.77022\n",
      "[63]\tvalid_0's auc: 0.770291\n",
      "Early stopping, best iteration is:\n",
      "[43]\tvalid_0's auc: 0.77085\n",
      "{'auc': 0.7758548016657666, 'logloss': 0.46030887404896165}\n"
     ]
    }
   ],
   "source": [
    "lgb_train = lgb.Dataset(train_x, train_y.reshape(-1), params=params)\n",
    "lgb_valid = lgb.Dataset(valid_x, valid_y.reshape(-1), reference=lgb_train)\n",
    "lgb_model = lgb.train(params,\n",
    "                      lgb_train,\n",
    "                      num_boost_round=NUM_OF_TREES,\n",
    "                      early_stopping_rounds=EARLY_STOPPING_ROUNDS,\n",
    "                      valid_sets=lgb_valid)\n",
    "test_preds = lgb_model.predict(test_x)\n",
    "auc = roc_auc_score(np.asarray(test_y.reshape(-1)), np.asarray(test_preds))\n",
    "logloss = log_loss(np.asarray(test_y.reshape(-1)), np.asarray(test_preds), eps=1e-12)\n",
    "res_optim = {\"auc\": auc, \"logloss\": logloss}\n",
    "print(res_optim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PY36-tf12",
   "language": "python",
   "name": "py36_b"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
